package Scraping;

/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */


import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class Scraping extends Thread {

    private BaseDatos5 base;
    private int contador = 0;
    private int contadorBD = 2;
    private String urlRaiz;

    public Scraping(String url) {
        base = new BaseDatos5(1);
        this.urlRaiz = url;
    }

    public void run() {
        base.add(this.urlRaiz);
        contador++;

        this.obtenerLinks(this.urlRaiz);

        while (true) {
            if (contador <= 5000) {
                String url2 = base.returnURL(contadorBD);
                this.obtenerLinks(url2);
                this.contadorBD++;
            } else {
                return;
            }
        }
    }

    private void obtenerLinks(String url) {
        try {
            Document doc = Jsoup.connect(url).get();
            
            //Limitar el contenido a la parte del texto central.
            Element content = doc.getElementById("mw-content-text");

            Elements elementos = content.children();

            for (Element elemento : elementos) {
                if (elemento.tagName().compareTo("p") == 0) {
                    Elements links = elemento.select("a[href]");

                    for (Element link : links) {
                        String l = "";
                        try {
                            System.out.println(link.attr("abs:href"));
                            l = link.attr("abs:href");
                            if (l.compareTo("") != 0) {
                                if (this.base.contains(l) == false) {
                                    this.base.add(l);
                                    this.contador++;
                                }
                            }
                        } catch (Exception e) {
                            System.out.println(e.toString());
                        }
                    }
                }
            }
        } catch (Exception ex) {
            System.out.println("Excepcion al abrir la URL en Scraping!!!!");
            System.out.println(ex.toString());
        }
    }
}
